import java.io.StringReader
import org.apache.lucene.analysis.standard.StandardAnalyzer
import org.apache.lucene.analysis.th.ThaiAnalyzer
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;

class AnalyzerTests extends GroovyTestCase {
    def getTokens = { analyzer, text ->
        def tokens = tokensFromAnalysis(analyzer, text)
        tokens.collect() {
            it.termText()
        }
    }
    def tokensFromAnalysis = { analyzer, text ->
        def stream = analyzer.tokenStream("contents", new StringReader(text))
        def list = []
        while (true) {
            def token = stream.next()
            if (token) {
                list << token
            }  else {
                break
            }
        }
       return list
    }

    void  testSimpleAnalyzer() {
        def stdAnalyzer = new StandardAnalyzer()

        def text = "The quick brown fox jumped over the lazy dogs"
        def tokens = getTokens(stdAnalyzer, text)
        def expects = ['quick', 'brown', 'fox', 'jumped', 'over', 'lazy', 'dogs']
        assertTrue(expects.equals(tokens))

        text = "XY&Z Corporation - xyz@example.com"
        tokens = getTokens(stdAnalyzer, text)
        expects = ['xy&z', 'corporation', 'xyz@example.com']
        assertTrue(expects.equals(tokens))        
    }

    void testThaiStopFilter() {
        def analyzer = new ThaiAnalyzerWithThaiStopFilter()
        def tokens = getTokens(analyzer, "ที่นี่ เมืองไทยเป็นเมืองพุทธ")
        def expects = ['เมือง', 'ไทย', 'เมือง', 'พุทธ']
        assertTrue(expects.equals(tokens))
    }

    void testSomeNameEntities() {
        def analyzer = new ThaiAnalyzer()

        def tokens = getTokens(analyzer, "ธรรมนิติ")
        assertTrue(['ธรรม', 'นิติ'].equals(tokens))

        tokens = getTokens(analyzer, "อนุสาวรีย์ประชาธิปไตย")
        assertTrue(['อนุสาวรีย์', 'ประชาธิปไตย'].equals(tokens))
    }

}

class ThaiAnalyzerWithThaiStopFilter  extends ThaiAnalyzer {
      public TokenStream tokenStream(String fieldName, Reader reader) {
	  def ts = super.tokenStream(fieldName, reader)
          return  new StopFilter(ts, ['เป็น', 'ที่', 'ก็', 'นี่'] as String[]);
      }
}
